{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1725672\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "import re\n",
    "\n",
    "input_ = open('/home/xuwenshen/data/big_data/raw_cases')\n",
    "output_ = open('/home/xuwenshen/data/big_data/2017_3_13/cases', 'w')\n",
    "\n",
    "\n",
    "pattern_symbol = re.compile(r'[。！；;（）]')\n",
    "pattern_defend = re.compile(r'被告|罪犯|犯|罪|非法|涉嫌')\n",
    "\n",
    "counter = 0\n",
    "while True:\n",
    "    \n",
    "    line = input_.readline()\n",
    "    if line == '':\n",
    "        break\n",
    "    line = json.loads(line)\n",
    "    \n",
    "    #reason begin\n",
    "    if len(line['reason']) > 250: continue\n",
    "    #reason end \n",
    "    \n",
    "    # source concate begin\n",
    "    source = ''\n",
    "    for i in range(len(line['source'])):\n",
    "        source += line['source'][i]\n",
    "    if len(source) > 1500: continue\n",
    "    # source concate end\n",
    "    \n",
    "    source_tmp = re.sub(pattern_symbol, '，', source)\n",
    "    source_tmp = source_tmp.split('，')\n",
    "    \n",
    "    # search defendant information begin\n",
    "    source_def = ''\n",
    "    defendant_candidate = []\n",
    "    for i in range(len(source_tmp)):\n",
    "        if re.search(pattern_defend, source_tmp[i]):\n",
    "            if len(source_tmp[i]) < 30:\n",
    "                source_def += source_tmp[i] + '，'\n",
    "            else:\n",
    "                defendant_candidate.append(source_tmp[i])\n",
    "    \n",
    "    for i in range(len(defendant_candidate)):\n",
    "        source_def += defendant_candidate[i] + '，'\n",
    "            \n",
    "    if len(source_def) > 300: continue\n",
    "          \n",
    "    # search defendant information end \n",
    "    \n",
    "    to_write = {'source':source, 'reason': line['reason'], 'defendant': source_def}\n",
    "    \n",
    "    output_.write(json.dumps(to_write) + '\\n')\n",
    "        \n",
    "    counter += 1\n",
    "\n",
    "input_.close()\n",
    "output_.close()\n",
    "print (counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1725672/1725672 [10:13<00:00, 2813.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8149\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "from tqdm import *\n",
    "\n",
    "input_ = open('/home/xuwenshen/data/big_data/2017_3_13/cases')\n",
    "\n",
    "raw_words = dict()\n",
    "\n",
    "\n",
    "for j in tqdm(range(1725672)):\n",
    "    line = input_.readline()\n",
    "    if line == '':\n",
    "        break\n",
    "    line = json.loads(line)\n",
    "    for i in range(len(line['source'])):\n",
    "        if line['source'][i] in raw_words:\n",
    "            raw_words[line['source'][i]] += 1\n",
    "        else:\n",
    "            raw_words.update({line['source'][i]:1})\n",
    "                \n",
    "    \n",
    "    for i in range(len(line['reason'])):\n",
    "        if line['reason'][i] in raw_words:\n",
    "            raw_words[line['reason'][i]] += 1\n",
    "        else:\n",
    "            raw_words.update({line['reason'][i]:1})\n",
    "    \n",
    "input_.close()\n",
    "\n",
    "print (len(raw_words))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "output_ =open('/home/xuwenshen/data/big_data/2017_3_13/tot_words', 'w')\n",
    "\n",
    "for word in raw_words:\n",
    "    output_.write(word + ' ' + str(raw_words[word]))\n",
    "    output_.write('\\n')\n",
    "output_.close()\n",
    "\n",
    "\n",
    "print ('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "input_ = open('/home/xuwenshen/data/big_data/2017_3_13/tot_words')\n",
    "words = dict()\n",
    "\n",
    "\n",
    "while True:\n",
    "    line = input_.readline()\n",
    "    if line == '':\n",
    "        break\n",
    "        \n",
    "    line = line.rstrip('\\n').split()\n",
    "    if len(line) != 2:\n",
    "        continue\n",
    "    words.update({line[0]:int(line[1])})\n",
    "\n",
    "input_.close()\n",
    "\n",
    "sorted_words = sorted(words.items(), key = lambda words:words[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2184\n"
     ]
    }
   ],
   "source": [
    "hwords = set()\n",
    "\n",
    "for i in range(len(sorted_words)):\n",
    "    if sorted_words[i][1] >= 5000:\n",
    "        hwords.add(sorted_words[i][0])\n",
    "\n",
    "\n",
    "hwords.add('eos')\n",
    "hwords.add('pad')\n",
    "hwords.add('go')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "output_ = open('/home/xuwenshen/data/big_data/2017_3_13/words', 'w')\n",
    "\n",
    "for word in hwords:\n",
    "    output_.write(word+'\\n')\n",
    "\n",
    "output_.close()\n",
    "\n",
    "print ('done')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1725672/1725672 [13:01<00:00, 2209.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from tqdm import *\n",
    "\n",
    "input_ = open('/home/xuwenshen/data/big_data/2017_3_13/cases')\n",
    "output_ = open('/home/xuwenshen/data/big_data/2017_3_13/simplified_unshuffled_cases', 'w')\n",
    "\n",
    "for j in tqdm(range(1725672)):\n",
    "    \n",
    "    line = input_.readline()\n",
    "    if line == '':\n",
    "        break\n",
    "    line = json.loads(line)\n",
    "\n",
    "    new_text = dict()\n",
    "    \n",
    "    source_tx = ''\n",
    "    for i in range(len(line['source'])):\n",
    "        if line['source'][i] in hwords:\n",
    "            source_tx += line['source'][i]\n",
    "    new_text.update({'source':source_tx})\n",
    "    \n",
    "    defendant_tx = ''\n",
    "    for i in range(len(line['defendant'])):\n",
    "        if line['defendant'][i] in hwords:\n",
    "            defendant_tx += line['defendant'][i]\n",
    "    new_text.update({'defendant':defendant_tx})\n",
    "    \n",
    "    reason_tx = ''\n",
    "    for i in range(len(line['reason'])):\n",
    "        if line['reason'][i] in hwords:\n",
    "            reason_tx += line['reason'][i]\n",
    "    new_text.update({'reason':reason_tx})\n",
    "    \n",
    "    output_.write(json.dumps(new_text))\n",
    "    output_.write('\\n')\n",
    "\n",
    "input_.close()\n",
    "output_.close()\n",
    "print ('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1725672/1725672 [00:04<00:00, 426794.12it/s]\n",
      "100%|██████████| 1725672/1725672 [00:43<00:00, 39477.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1725672\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "from tqdm import *\n",
    "\n",
    "input_ = open('/home/xuwenshen/data/big_data/2017_3_13/simplified_unshuffled_cases')\n",
    "\n",
    "datas = [_.rstrip('\\n') for _ in tqdm(input_.readlines())]\n",
    "\n",
    "random.shuffle(datas)\n",
    "\n",
    "input_.close()\n",
    "\n",
    "output_ = open('/home/xuwenshen/data/big_data/2017_3_13/shuffled_cases', 'w')\n",
    "\n",
    "for i in tqdm(range(len(datas))):\n",
    "    output_.write(datas[i])\n",
    "    output_.write('\\n')\n",
    "\n",
    "output_.close()\n",
    "\n",
    "print (len(datas))\n",
    "print ('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1600000/1600000 [03:53<00:00, 6852.32it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "from tqdm import *\n",
    "\n",
    "input_ = open('/home/xuwenshen/data/big_data/2017_3_13/shuffled_cases')\n",
    "\n",
    "output_ = open('/home/xuwenshen/data/big_data/2017_3_13/shuffled_train', 'w')\n",
    "\n",
    "for i in tqdm(range(1600000)):\n",
    "    line = input_.readline()\n",
    "    output_.write(line)\n",
    "\n",
    "output_.close()\n",
    "\n",
    "output_ = open('/home/xuwenshen/data/big_data/2017_3_13/shuffled_test', 'w')\n",
    "\n",
    "for i in range(1600000, 1725672):\n",
    "    line = input_.readline()\n",
    "    output_.write(line)\n",
    "\n",
    "output_.close()\n",
    "input_.close()\n",
    "\n",
    "print ('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10000/10000 [00:00<00:00, 90072.05it/s]\n",
      "100%|██████████| 115672/115672 [00:01<00:00, 78454.18it/s]\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import re\n",
    "\n",
    "\n",
    "input_ = open('/home/xuwenshen/data/big_data/2017_3_13/shuffled_test')\n",
    "\n",
    "output_ = open('/home/xuwenshen/data/big_data/2017_3_13/shuffled_valid', 'w')\n",
    "\n",
    "for i in tqdm(range(10000)):\n",
    "    output_.write(input_.readline())\n",
    "\n",
    "output_.close()\n",
    "\n",
    "output_ = open('/home/xuwenshen/data/big_data/2017_3_13/shuffled_test1', 'w')\n",
    "\n",
    "for i in tqdm(range(10000, 125672)):\n",
    "    output_.write(input_.readline())\n",
    "\n",
    "output_.close()\n",
    "input_.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
